package com.it.spider;
/*
* Created by liuyonghui on 2015/12/14.
*
* 用于抓取全国学校信息
* */
import com.gargoylesoftware.htmlunit.*;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import com.gargoylesoftware.htmlunit.util.NameValuePair;
import com.it.domain.Bd_schooltemp;
import com.it.domain.School;
import com.it.utils.MyBatisUtil;
import com.it.utils.RandomCodeUtil;
import net.sf.json.JSONArray;
import net.sf.json.JSONException;
import net.sf.json.JSONObject;
import org.apache.ibatis.session.SqlSession;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;

import java.awt.*;
import java.io.IOException;
import java.net.URL;
import java.sql.SQLException;
import java.util.*;
import java.util.List;

public class RetrivePage {

    /**
     * @param args
     */
    /******
     * 抓取学校信息
     * www.tizi.com
     * ********/
    public  void schoolInf() {
        //int id=0;
        int count =0;
        try {
            /*拿到省名列表*/
            Document doc_p = Jsoup.connect("http://www.tizi.com/class/area?id=1").get();
            //System.out.println(doc_p.text());
            // 将json字符串转换为json对象
            JSONArray jsonObject_p = JSONArray.fromObject(doc_p.text());

            for (int i = jsonObject_p.size()-1; i <jsonObject_p.size(); i++) {
                //拿到各省id 名称
                String id_province = jsonObject_p.getJSONObject(i).getString("id");
                String province = jsonObject_p.getJSONObject(i).getString("name");
                //各市
                Document doc_c = Jsoup.connect("http://www.tizi.com/class/area?id=" + id_province).get();
                JSONArray jsonObject_city = JSONArray.fromObject(doc_c.text());
                for (int j = 0; j < jsonObject_city.size();j++) {

                    String level2_id = jsonObject_city.getJSONObject(j).getString("id");
                    String city = jsonObject_city.getJSONObject(j).getString("name");
                    //行政级别（北京1 城东区3 ·陕西1 西安2）
                    String level = jsonObject_city.getJSONObject(j).getString("level");
                    //直辖市
                    if("3".equals(level)) {
                        System.out.println(province +city);
                        String area =province+"市"+city;
                        String county= city;
                        List<Bd_schooltemp> schoolList =new ArrayList();
                        getSchool(level2_id,"1",area,schoolList,province,null,county);
                        getSchool(level2_id,"2",area,schoolList,province,null,county);
                        System.out.println(area+"： 共计学校："+schoolList.size());
                        if(schoolList.size()>0) {
                            //将一个县区学校批插入到数据库中
                            System.out.println("*******************正在插入" + province + "市" + city + "学校信息");
                            batchAdd(schoolList);
                            System.out.println("*******************已经在插入" + province + "市" + city + "学校信息");
                        }

                    }
                    //各省区县
                   if("2".equals(level)){
                        Document doc_county = Jsoup.connect("http://www.tizi.com/class/area?id="+level2_id).get();
                        JSONArray jsonObject_county =JSONArray.fromObject(doc_county.text());
                        for(int z =0; z<jsonObject_county.size() ;z++){
                            String county = jsonObject_county.getJSONObject(z).getString("name");
                            String county_id = jsonObject_county.getJSONObject(z).getString("id");
                            System.out.println(province + city +county);
                            String area=province+"省" + city+"市" +county;
                            List<Bd_schooltemp> schoolList =new ArrayList();
                                    getSchool(county_id,"1",area,schoolList,province,city ,county);
                                    getSchool(county_id,"2",area,schoolList,province,city,county);
                            System.out.println(area+"： 共计学校："+schoolList.size());
                            if(schoolList.size()>0) {
                                //将一个县区学校批插入到数据库中
                                System.out.println("*******************正在插入"+area+"学校信息");
                                try {
                                    batchAdd(schoolList);
                                }catch(Exception e){
                                    System.out.println(area+"地区学校未批插入，正在尝试逐个插入。。。。。。。。");
                                    for(Bd_schooltemp school:schoolList) {
                                        try {
                                            add(school);
                                           // System.out.println(area+"地区学校未批插入，逐个插入。。。。。。。。");
                                        } catch (Exception c) {
                                             continue;
                                        }
                                    }

                                }
                            }

                        }

                    }


                }

            }
        } catch (IOException e) {
            e.printStackTrace();
        }
    }



    public void  getSchool(String id, String scype,String area,List<Bd_schooltemp> schoolList,String province,String city ,String county){
        HtmlPage scorePage = null;
        WebClient webclient = null;
        webclient = new WebClient(BrowserVersion.CHROME);
        webclient.setAjaxController(new NicelyResynchronizingAjaxController());
        WebClientOptions options = webclient.getOptions();
        webclient.waitForBackgroundJavaScript(1000);
        options.setJavaScriptEnabled(true);
        options.setRedirectEnabled(true);
        options.setActiveXNative(false);
        options.setAppletEnabled(false);
        options.setCssEnabled(false);
        options.setDoNotTrackEnabled(false);
        options.setGeolocationEnabled(false);
        options.setUseInsecureSSL(true);
        options.setThrowExceptionOnFailingStatusCode(false);
        options.setThrowExceptionOnScriptError(false);

        try {

            HtmlPage resultPage = null;
            WebRequest request = new WebRequest(new URL("http://www.tizi.com/class/schools/county_sch"));
            request.setHttpMethod(HttpMethod.POST);

            List<NameValuePair> requestParameters = new ArrayList();
            requestParameters.add(new NameValuePair("id", id));
            requestParameters.add(new NameValuePair("sctype",  scype));
            requestParameters.add(new NameValuePair("token", "2910d526764def18d2cc10247c3aed34"));
            requestParameters.add(new NameValuePair("page_name", "1492a797df4708f6c25bca7a683599b8"));
            requestParameters.add(new NameValuePair("v", "1450316872914"));
            request.setRequestParameters(requestParameters);

            Map<String, String> head = new HashMap();
           // head.put("Cookie", "TZID=fe83ef6a729b6011bcb8c7e819fd6700;uid=eccUO1Zv45uogAM6JC8JAg==");
            head.put("Cookie", "TZID=c3c32103b3e8dc3fdb96373081254407;uid=eccUO1ZxZKqoWwM5JbLAAg==;");
            head.put("Referer", "http://www.tizi.com/banji");
            head.put("Host", "www.tizi.com");
            request.setAdditionalHeaders(head);
            resultPage = webclient.getPage(request);
           //System.out.println(resultPage.asText());
           ;

            String result =resultPage.asText();
            //解析获取到的json
            JSONObject schoolJson =JSONObject.fromObject(resultPage.asText());
          //  System.out.println(schoolJson);
            String data = schoolJson.getString("data");
            System.out.println("抓取到的json："+data);
         if(!data.startsWith("[")) {
                JSONObject dataJson =  JSONObject.fromObject(data);
                System.out.println(dataJson);

             //去重复数据
             Map<String,Bd_schooltemp> schoolMap =new HashMap();
                    for (int j = 97; j <= 122; j++) {
                        String code = null;
                        System.out.println();
                        try {
                            code = dataJson.getString((char) j + "");
                          /// System.out.println(code);
                        } catch (JSONException e) {
                            System.out.println((char) j + "   开头学校无");
                        }
                        if (code == null)
                            continue;
                        //System.out.println("hahaha         "  +(char) j+"");
                        JSONArray schoolArray =  JSONArray.fromObject(code );
                        //System.out.println(schoolArray.length());
                        int count=0;

                        for (int z = 0; z < schoolArray.size(); z++) {
                            String school_id = schoolArray.getJSONObject(z).getString("id");
                            String school_name = schoolArray.getJSONObject(z).getString("schoolname");

                            if ("其他".equals(school_name))
                                continue;
                            //String school_type;
                            /** 学校办学类型 */
                            String schoolbxlx;
                            /** 学校办别 */
                             String schoolxxbb="公立";
                            /** 学校类别 */
                             String schoolxxlb;

                            if (scype == "1") {
                                schoolbxlx = "公立小学";
                                schoolxxlb ="小学";
                            } else {
                                schoolbxlx = "公立中学";
                                schoolxxlb ="中学";
                            }
                            String py = schoolArray.getJSONObject(z).getString("py");
                            String  schoolcode =py.substring(0,1)  +RandomCodeUtil.generateNumCode(8);

                            System.out.println("school" + school_id + "   " + school_name + " " + area + " " + schoolbxlx);
                            Bd_schooltemp school = new Bd_schooltemp();
                            school.setCode(schoolcode);
                            school.setName(school_name);
                            school.setAddress_province(province);
                            school.setAddress_city(city);
                            school.setAddress_county(county);
                            //学校办学类别
                            school.setSchoolbxlx(schoolbxlx);
                            //学校办别
                            school.setSchoolxxbb(schoolxxbb);
                            //学校类别
                            school.setSchoolxxlb(schoolxxlb);
                          //  school.setSchool_area(area);
                          /*  school.setSchool_type(school_type);
                            school.setSchool_province(province);
                            school.setSchool_city(city);
                            school.setSchool_county(county);*/
                            schoolMap.put(school.getName(),school);
                            //schoolList.add(school);
                            //System.out.println(school.getSchool_area());
                            //插入数据库
                            //add(school);
                        }

                    }
             for(String key:schoolMap.keySet() ){
                 schoolList.add(schoolMap.get(key));
                // System.out.println(schoolMap.get(key).getAddress_county());
             }

                }


        } catch (IOException e) {
            e.printStackTrace();
        }


    }
    /*
    * 将数据插入数据库
    * */
    public void add(Bd_schooltemp school){
        //SqlSession sqlSession = MyBatisUtil.getSqlSession(false);
        SqlSession sqlSession = MyBatisUtil.getSqlSession(true);
        /**
         * 映射sql的标识字符串，
         * cn.it.mapping.userMapper是userMapper.xml文件中mapper标签的namespace属性的值，
         * addUser是insert标签的id属性值，通过insert标签的id属性值就可以找到要执行的SQL
         */

        String statement = "com.it.mapping.bd_schooltempMapper.addSchool";//映射sql的标识字符串

        //执行插入操作
        int retResult = sqlSession.insert(statement,school);

        //手动提交事务
        //sqlSession.commit();
        //使用SqlSession执行完SQL之后需要关闭SqlSession
        sqlSession.close();
        System.out.println(retResult);
    }
    public void batchAdd(List<Bd_schooltemp> schoolList){
        //SqlSession sqlSession = MyBatisUtil.getSqlSession(false);
        SqlSession sqlSession = MyBatisUtil.getSqlSession(true);
        /**
         * 映射sql的标识字符串，
         * cn.it.mapping.userMapper是userMapper.xml文件中mapper标签的namespace属性的值，
         * addUser是insert标签的id属性值，通过insert标签的id属性值就可以找到要执行的SQL
         */
        //执行插入操作
        int  retResult =sqlSession.insert("com.it.mapping.bd_schooltempMapper.batchInsertSchool", schoolList);
        //手动提交事务
        //sqlSession.commit();
        //使用SqlSession执行完SQL之后需要关闭SqlSession
        sqlSession.close();
        System.out.println(retResult);

    }


}
